{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9577819-100c-4fc3-8b0c-79cc016658f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import zipfile\n",
    "import io\n",
    "import os\n",
    "\n",
    "def download_and_extract_dataset(url, output_dir):\n",
    "    if not os.path.exists(output_dir):\n",
    "        os.makedirs(output_dir)\n",
    "    \n",
    "    response = requests.get(url)\n",
    "    zip_content = zipfile.ZipFile(io.BytesIO(response.content))\n",
    "    zip_content.extractall(output_dir)\n",
    "\n",
    "# 下载MovieLens数据集\n",
    "movie_lens_url = 'http://files.grouplens.org/datasets/movielens/ml-latest-small.zip'\n",
    "output_dir = './data'\n",
    "download_and_extract_dataset(movie_lens_url, output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce2ff0db-a836-4358-afbd-de05213ff695",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "class DBSCAN:\n",
    "    def __init__(self, eps, min_pts):\n",
    "        self.eps = eps\n",
    "        self.min_pts = min_pts\n",
    "        self.labels_ = []\n",
    "\n",
    "    def fit(self, X):\n",
    "        nbrs = NearestNeighbors(n_neighbors=self.min_pts).fit(X)\n",
    "        distances, indices = nbrs.kneighbors(X)\n",
    "        \n",
    "        core_samples_mask = np.zeros_like(distances[:, 0], dtype=bool)\n",
    "        labels = np.full(X.shape[0], -1)\n",
    "        cluster_id = 0\n",
    "        \n",
    "        for i in range(X.shape[0]):\n",
    "            if labels[i] != -1:\n",
    "                continue\n",
    "            \n",
    "            neighbors = indices[i, 1:]  # ignore the point itself\n",
    "            if len(neighbors) < self.min_pts:\n",
    "                labels[i] = 0  # Noise\n",
    "            else:\n",
    "                core_samples_mask[i] = True\n",
    "                labels[i] = cluster_id\n",
    "                \n",
    "                for neighbor in neighbors:\n",
    "                    if labels[neighbor] == -1:\n",
    "                        labels[neighbor] = cluster_id\n",
    "                    elif labels[neighbor] != 0 and labels[neighbor] != cluster_id:\n",
    "                        raise ValueError(\"Found different clusters assigned to the same core sample.\")\n",
    "                \n",
    "                cluster_id += 1\n",
    "        \n",
    "        self.labels_ = labels\n",
    "\n",
    "    def predict(self):\n",
    "        return self.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a88ded7-c20d-48b1-85e9-b83596363882",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.cluster import DBSCAN\n",
    "\n",
    "# 生成虚拟数据集\n",
    "X, _ = make_blobs(n_samples=300, centers=4, cluster_std=0.60, random_state=0)\n",
    "\n",
    "# 绘制k-Distance图以确定eps值\n",
    "def plot_k_distance(X, k=4):\n",
    "    nbrs = NearestNeighbors(n_neighbors=k+1).fit(X)\n",
    "    distances, _ = nbrs.kneighbors(X)\n",
    "    k_distances = distances[:, k]\n",
    "    sorted_indices = np.argsort(k_distances)\n",
    "    plt.plot(sorted_indices, k_distances[sorted_indices])\n",
    "    plt.ylabel('k-th Nearest Neighbor Distance')\n",
    "    plt.xlabel('Data points sorted by distance')\n",
    "    plt.show()\n",
    "\n",
    "# 绘制k-Distance图\n",
    "plot_k_distance(X)\n",
    "\n",
    "# 选择eps值（例如通过观察k-Distance图中的拐点）\n",
    "eps = 0.3  # 根据k-Distance图选择\n",
    "min_pts = 5\n",
    "\n",
    "# 使用DBSCAN进行聚类\n",
    "dbscan = DBSCAN(eps=eps, min_samples=min_pts)\n",
    "dbscan.fit(X)\n",
    "labels = dbscan.labels_\n",
    "\n",
    "# 可视化聚类结果\n",
    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7ec90f8-dd75-48fc-a56d-cace2cecfef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.cluster import DBSCAN\n",
    "\n",
    "# 读取MovieLens数据集\n",
    "ratings = pd.read_csv('./data/ml-latest-small/ratings.csv')\n",
    "movies = pd.read_csv('./data/ml-latest-small/movies.csv')\n",
    "\n",
    "# 数据预处理\n",
    "ratings = ratings.dropna()\n",
    "user_movie_matrix = ratings.pivot(index='userId', columns='movieId', values='rating').fillna(0)\n",
    "\n",
    "# 处理异常值（如将极端评分进行平滑处理）\n",
    "def smooth_extreme_ratings(matrix, threshold=4.5):\n",
    "    return np.clip(matrix, a_min=threshold-1, a_max=threshold).astype(float)  # 保持数据类型为float\n",
    "\n",
    "user_movie_matrix = smooth_extreme_ratings(user_movie_matrix)  # 注意这里不需要.values，因为pivot已经返回了DataFrame\n",
    "\n",
    "# 但是，由于DBSCAN通常用于稀疏数据或特征向量，而不是整个用户-项目矩阵，\n",
    "# 我们可能需要将用户-项目矩阵转换为用户或项目的特征向量。\n",
    "# 这里为了简单起见，我们仍然使用整个矩阵，但请注意这通常不是DBSCAN的最佳用例。\n",
    "\n",
    "# 将DataFrame转换为二维NumPy数组（如果它还不是）\n",
    "user_movie_matrix_array = user_movie_matrix.values\n",
    "\n",
    "# 使用DBSCAN进行聚类\n",
    "eps = 0.5  # 需要根据数据特点调整\n",
    "min_samples = 5  # 注意参数名应该是min_samples，而不是min_pts\n",
    "\n",
    "dbscan = DBSCAN(eps=eps, min_samples=min_samples)  # 使用关键字参数\n",
    "user_labels = dbscan.fit_predict(user_movie_matrix_array)\n",
    "\n",
    "# 分析聚类结果\n",
    "num_clusters = len(set(user_labels)) - (1 if -1 in user_labels else 0)\n",
    "print(f\"Number of clusters: {num_clusters}\")\n",
    "plt.hist(user_labels, bins=range(-1, num_clusters+2), edgecolor='black')\n",
    "plt.xlabel('Cluster Label')\n",
    "plt.ylabel('Number of Users')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ad42bfb-4c0d-4487-a8a5-dce327271f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "class UserBasedCollaborativeFiltering:\n",
    "    def __init__(self, similarity_metric=cosine_similarity):\n",
    "        self.similarity_metric = similarity_metric\n",
    "\n",
    "    def fit(self, user_item_matrix):\n",
    "        # 确保没有NaN值\n",
    "        self.user_item_matrix = user_item_matrix.fillna(0)\n",
    "        self.similarity_matrix = self.similarity_metric(self.user_item_matrix)\n",
    "\n",
    "    def predict(self, user_id, movie_id):\n",
    "        user_index = self.user_item_matrix.index.get_loc(user_id)  # 使用.get_loc()查找索引\n",
    "        movie_index = self.user_item_matrix.columns.get_loc(movie_id)  # 使用.get_loc()查找列索引\n",
    "        \n",
    "        # ...（其余代码逻辑不变，但使用正确的索引）\n",
    "\n",
    "# 假设ratings和movies已经被正确加载和预处理\n",
    "# ...（加载和预处理代码）\n",
    "\n",
    "# 实例化协同过滤模型并拟合数据\n",
    "cf_model = UserBasedCollaborativeFiltering()\n",
    "cf_model.fit(user_movie_matrix)\n",
    "\n",
    "# 选择一个用户进行评分预测（这里不使用聚类结果，仅为示例）\n",
    "target_user_id = ratings.iloc[0]['userId']  # 选择第一个用户作为目标用户\n",
    "\n",
    "# 找到目标用户未观看的电影进行评分预测\n",
    "unwatched_movies_indices = np.where(user_movie_matrix.loc[target_user_id] == 0)[0]\n",
    "predictions = []\n",
    "\n",
    "for movie_index in unwatched_movies_indices[:5]:  # 预测前5部未观看电影的评分\n",
    "    movie_id = user_movie_matrix.columns[movie_index]\n",
    "    predicted_rating = cf_model.predict(target_user_id, movie_id)\n",
    "    predictions.append((movie_id, predicted_rating))\n",
    "\n",
    "predictions_df = pd.DataFrame(predictions, columns=['movieId', 'predictedRating'])\n",
    "print(predictions_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
